{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import time\n",
    "import warnings\n",
    "import lightgbm as lgb;\n",
    "warnings.simplefilter(action = 'ignore', category = FutureWarning)\n",
    "from scipy import stats;\n",
    "from sklearn.metrics import roc_auc_score, precision_score, recall_score\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "from bayes_opt import BayesianOptimization;\n",
    "from lightgbm import LGBMClassifier\n",
    "from scipy.stats import ranksums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt = pd.read_csv(\"../input/ds9.csv\") # loading our pooled features, about 2k features\n",
    "\n",
    "\n",
    "CATEGORICAL_COLUMNS = ['CODE_GENDER',\n",
    "                       'EMERGENCYSTATE_MODE',\n",
    "                       'FLAG_CONT_MOBILE',\n",
    "                       'FLAG_DOCUMENT_3',\n",
    "                       'FLAG_DOCUMENT_4',\n",
    "                       'FLAG_DOCUMENT_5',\n",
    "                       'FLAG_DOCUMENT_6',\n",
    "                       'FLAG_DOCUMENT_7',\n",
    "                       'FLAG_DOCUMENT_8',\n",
    "                       'FLAG_DOCUMENT_9',\n",
    "                       'FLAG_DOCUMENT_11',\n",
    "                       'FLAG_DOCUMENT_18',\n",
    "                       'FLAG_EMAIL',\n",
    "                       'FLAG_EMP_PHONE',\n",
    "                       'FLAG_MOBIL',\n",
    "                       'FLAG_OWN_CAR',\n",
    "                       'FLAG_OWN_REALTY',\n",
    "                       'FLAG_PHONE',\n",
    "                       'FLAG_WORK_PHONE',\n",
    "                       'FONDKAPREMONT_MODE',\n",
    "                       'HOUR_APPR_PROCESS_START',\n",
    "                       'HOUSETYPE_MODE',\n",
    "                       'LIVE_CITY_NOT_WORK_CITY',\n",
    "                       'LIVE_REGION_NOT_WORK_REGION',\n",
    "                       'NAME_CONTRACT_TYPE',\n",
    "                       'NAME_TYPE_SUITE',\n",
    "                       'NAME_INCOME_TYPE',\n",
    "                       'NAME_EDUCATION_TYPE',\n",
    "                       'NAME_FAMILY_STATUS',\n",
    "                       'NAME_HOUSING_TYPE',\n",
    "                       'OCCUPATION_TYPE',\n",
    "                       'ORGANIZATION_TYPE',\n",
    "                       'REG_CITY_NOT_LIVE_CITY',\n",
    "                       'REG_CITY_NOT_WORK_CITY',\n",
    "                       'REG_REGION_NOT_LIVE_REGION',\n",
    "                       'REG_REGION_NOT_WORK_REGION',\n",
    "                       'WALLSMATERIAL_MODE',\n",
    "                       'WEEKDAY_APPR_PROCESS_START']\n",
    "\n",
    "\n",
    "train_df = dt.iloc[:307511,]\n",
    "test_df = dt.iloc[307511:,]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_id = train_df.SK_ID_CURR\n",
    "test_id = test_df.SK_ID_CURR\n",
    "\n",
    "train_df.drop([\"TARGET.x\",\"SK_ID_CURR\"],axis=1,inplace = True)\n",
    "test_df.drop([\"TARGET.x\",\"SK_ID_CURR\"],axis=1,inplace = True)\n",
    "\n",
    "fold_df = pd.read_csv('../input/folds.csv') # importing folds (actually generated by sklearn KFold class with shuffle=True and seed=1001 then dumped to a csv so all team members use the same fold indices for creationg new oofs)\n",
    "Y = fold_df.Y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Actually it is not for parameter optimiztion, it is for creation of new and different oofs; But how ? \n",
    "\n",
    "well; every now and then we dump a good candidate solution as an oof...\n",
    "You can change it to work with Catboost or XGBoost easily"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lgb_eval(num_leaves, feature_fraction, bagging_fraction, max_depth, lambda_l1, lambda_l2, min_split_gain, min_child_weight):\n",
    "    params = {'application':'binary','num_iterations':4000, 'learning_rate':0.05, 'early_stopping_round':100, 'metric':'auc'}\n",
    "    params[\"num_leaves\"] = round(num_leaves)\n",
    "    params['feature_fraction'] = max(min(feature_fraction, 1), 0)\n",
    "    params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)\n",
    "    params['max_depth'] = round(max_depth)\n",
    "    params['lambda_l1'] = max(lambda_l1, 0)\n",
    "    params['lambda_l2'] = max(lambda_l2, 0)\n",
    "    params['min_split_gain'] = min_split_gain\n",
    "    params['min_child_weight'] = min_child_weight\n",
    "    cv_result = lgb_cv(num_leaves, feature_fraction, bagging_fraction, max_depth, lambda_l1, lambda_l2, min_split_gain, min_child_weight)\n",
    "    return cv_result;\n",
    "\n",
    "def lgb_cv(num_leaves, feature_fraction, bagging_fraction, max_depth, lambda_l1, lambda_l2, min_split_gain, min_child_weight):\n",
    "    Y =  np.array(fold_df.Y);\n",
    "    oof_preds = np.zeros(train_df.shape[0])\n",
    "    sub_preds = np.zeros(test_df.shape[0])\n",
    "    for i in range(1,6):\n",
    "        for j in [0]:\n",
    "            ft_to_use = train_df.columns.values;\n",
    "            CATEGORICAL_COLUMNS_X = [f for f in ft_to_use if f in CATEGORICAL_COLUMNS]\n",
    "            dtest = lgb.Dataset(data=test_df, \n",
    "                                categorical_feature = CATEGORICAL_COLUMNS_X,\n",
    "                                free_raw_data=False, silent=True)\n",
    "            ix_valid = fold_df[fold_df.FoldID==i].index;\n",
    "            ix_train = fold_df[fold_df.FoldID!=i].index;\n",
    "            x_train = train_df.iloc[ix_train];\n",
    "            x_train = x_train;\n",
    "            v_train = train_df.iloc[ix_valid];\n",
    "            v_train = v_train;\n",
    "    \n",
    "            y_train = Y[ix_train];\n",
    "            y_valid = Y[ix_valid];\n",
    "            dtrain = lgb.Dataset(data=x_train, \n",
    "                                 label=y_train, \n",
    "                                 categorical_feature = CATEGORICAL_COLUMNS_X,\n",
    "                                 free_raw_data=False, silent=True)\n",
    "            dvalid = lgb.Dataset(data=v_train, \n",
    "                                 label=y_valid, \n",
    "                                 categorical_feature = CATEGORICAL_COLUMNS_X,\n",
    "                                 free_raw_data=False, silent=True)\n",
    "        \n",
    "            #multiple runs\n",
    "            params = {\n",
    "                    'objective': 'binary',\n",
    "                    'boosting_type': 'gbdt',\n",
    "                    'nthread': 10,\n",
    "                    'learning_rate': 0.02,  # 02,\n",
    "                    'num_leaves': int(num_leaves),\n",
    "                    'colsample_bytree': feature_fraction,\n",
    "                    'subsample': bagging_fraction,\n",
    "                    'subsample_freq': 1,\n",
    "                    'max_depth': int(max_depth),\n",
    "                    'reg_alpha': lambda_l1,\n",
    "                    'reg_lambda': lambda_l2,\n",
    "                    'min_split_gain': min_split_gain,\n",
    "                    'min_child_samples': int(min_child_weight), # 39.3259775,\n",
    "                    'seed': 0,\n",
    "                    'verbose': -1,\n",
    "                    'metric': 'auc',\n",
    "            }\n",
    "            clf = lgb.train(\n",
    "                params=params,\n",
    "                train_set=dtrain,\n",
    "                num_boost_round=100000,\n",
    "                categorical_feature = CATEGORICAL_COLUMNS_X,\n",
    "                valid_sets=[dtrain, dvalid],\n",
    "                early_stopping_rounds=200,\n",
    "                verbose_eval=500\n",
    "            );\n",
    "            oof_preds[ix_valid] += clf.predict(dvalid.data)\n",
    "            sub_preds += clf.predict(dtest.data);\n",
    "            sc = roc_auc_score(y_valid, oof_preds[ix_valid]);\n",
    "\n",
    "    #full test\n",
    "    oof_preds = oof_preds;\n",
    "    sub_preds = sub_preds / 5;   \n",
    "    \n",
    "    sc = roc_auc_score(Y, oof_preds);\n",
    "    submission_file_name = \"test_\" + str.replace(str(sc),'.','_') + \".csv\";\n",
    "    oof_file_name = \"oof_\" + str.replace(str(sc),'.','_') + \".csv\";\n",
    "    sub_df = pd.DataFrame(columns=[\"SK_ID_CURR\"],data=test_id);\n",
    "    sub_df['TARGET'] = sub_preds\n",
    "    sub_df[['SK_ID_CURR', 'TARGET']].to_csv(submission_file_name, index= False)\n",
    "    oof_df = pd.DataFrame(columns=[\"SK_ID_CURR\"],data=train_id);\n",
    "    oof_df['TARGET'] = oof_preds\n",
    "    oof_df[['SK_ID_CURR', 'TARGET']].to_csv(oof_file_name, index= False)\n",
    "    print('Full AUC score %.6f' % roc_auc_score(Y, oof_preds))            \n",
    "    x_train  = None;\n",
    "    v_train = None;\n",
    "    dtrain  = None;\n",
    "    dvalid = None;\n",
    "    dtest = None;\n",
    "    gc.collect();\n",
    "    return sc;\n",
    "\n",
    "\n",
    "gbBO = BayesianOptimization(lgb_eval, {'num_leaves': (18, 40),\n",
    "                                        'feature_fraction': (0.02, 0.2),\n",
    "                                        'bagging_fraction': (0.8, 1),\n",
    "                                        'max_depth': (-1,8),\n",
    "                                        'lambda_l1': (0, 100),\n",
    "                                        'lambda_l2': (0, 100),\n",
    "                                        'min_split_gain': (0.0, 0.5),\n",
    "                                        'min_child_weight': (10, 100)}, \n",
    "                                        random_state=0)\n",
    "            \n",
    "            \n",
    "gbBO.maximize(init_points=15, n_iter=100);\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
